Tensorflow Tutorial (writen by experience)

Alex Fang

1.Basic Structure

In [2]:
import tensorflow as tf
import numpy as np
import pandas as pd
import warnings
warnings.filterwarnings('ignore')
  • 1.tf.reset_default_graph()
  • 2.variables
  • 3.placeholder, input and output
  • 4.operation, the algorithms
  • 5.initialization
  • 6.sess.run(Variables),repeat call for variables, and it will do corresponding operations
  • 7.care about the train,test loss
  • 8.care about the accuracy, or mse...
In [44]:
#If you didn't set it, you will use all gpu automaticlly.
#os.environ['CUDA_VISIBLE_DEVICES'] = '0'

#clear all
tf.reset_default_graph()

#define the variable
# a good habit is to define the name, and name scope
with tf.name_scope("algoritm1_variable"):
    var1 = tf.Variable(np.zeros((2,3)),dtype=tf.float64,name="variable1")
    var1 = tf.Variable(np.zeros((4,3)),dtype=tf.float64,name="variable1")
    var2 = tf.Variable(np.zeros((4,3)),dtype=tf.float64,name="variable1")
print(var1)
print(var2)

#one graph
sess=tf.Session()

#initialization,very important
sess.run(tf.global_variables_initializer())    

#put the number into it 
var1_=sess.run(var1)
print(var1_)

<tf.Variable 'algoritm1_variable/variable1_1:0' shape=(4, 3) dtype=float64_ref>
<tf.Variable 'algoritm1_variable/variable1_2:0' shape=(4, 3) dtype=float64_ref>
[[0. 0. 0.]
 [0. 0. 0.]
 [0. 0. 0.]
 [0. 0. 0.]]
In [32]:
tf.reset_default_graph()
var1 = tf.constant(np.zeros((2,3)),dtype=tf.float64)
print(var1)

#one graph
sess=tf.Session()

#initialization,very important
sess.run(tf.global_variables_initializer())    

#put the number into it 
var1_=sess.run(var1)
print(var1_)

Tensor("Const:0", shape=(2, 3), dtype=float64)
[[0. 0. 0.]
 [0. 0. 0.]]
In [4]:
tf.reset_default_graph()

#as input or output
features=84
#Why we set None? because sometimes we want to use different number of samples to train.
tf_x = tf.placeholder(tf.float32, [None,features])

print(tf_x)

Tensor("Placeholder:0", shape=(?, 84), dtype=float32)

Example: Regression Problem

In [5]:
import tensorflow as tf
import matplotlib.pyplot as plt
import numpy as np

tf.set_random_seed(1)
np.random.seed(1)

# fake data
x = np.linspace(-1, 1, 100)[:, np.newaxis]          # shape (100, 1)
noise = np.random.normal(0, 0.1, size=x.shape)
y = np.power(x, 2) + noise                          # shape (100, 1) + some noise

# plot data
plt.scatter(x, y)
plt.show()

tf_x = tf.placeholder(tf.float32, x.shape)     # input x
tf_y = tf.placeholder(tf.float32, y.shape)     # input y

# neural network layers
l1 = tf.layers.dense(tf_x, 10, tf.nn.relu)          # hidden layer
output = tf.layers.dense(l1, 1)                     # output layer

loss = tf.losses.mean_squared_error(tf_y, output)   # compute cost
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.5)
train_op = optimizer.minimize(loss)

sess = tf.Session()                                 # control training and others
sess.run(tf.global_variables_initializer())         # initialize var in graph

plt.ion()   # something about plotting

for step in range(100):
    # train and net output
    _, l, pred = sess.run([train_op, loss, output], {tf_x: x, tf_y: y})
    if step % 20 == 0:
        print(l)
        # plot and show learning process
        plt.cla()
        plt.scatter(x, y)
        plt.plot(x, pred, 'r-', lw=5)
        plt.text(0.5, 0, 'Loss=%.4f' % l, fontdict={'size': 20, 'color': 'red'})
        plt.pause(0.1)

plt.ioff()
plt.show()

0.13032648

0.017207863

0.013244632

0.011768642

0.010693824

mass.gif

Example: Classification Problem

In [6]:
tf.set_random_seed(1)
np.random.seed(1)

# fake data
n_data = np.ones((100, 2))
x0 = np.random.normal(2*n_data, 1)      # class0 x shape=(100, 2)
y0 = np.zeros(100)                      # class0 y shape=(100, )
x1 = np.random.normal(-2*n_data, 1)     # class1 x shape=(100, 2)
y1 = np.ones(100)                       # class1 y shape=(100, )
x = np.vstack((x0, x1))  # shape (200, 2) + some noise
y = np.hstack((y0, y1))  # shape (200, )

# plot data
plt.scatter(x[:, 0], x[:, 1], c=y, s=100, lw=0, cmap='RdYlGn')
plt.show()

tf_x = tf.placeholder(tf.float32, x.shape)     # input x
tf_y = tf.placeholder(tf.int32, y.shape)     # input y

# neural network layers
l1 = tf.layers.dense(tf_x, 10, tf.nn.relu)          # hidden layer
output = tf.layers.dense(l1, 2)                     # output layer

#it is hard label here. but in the reality, reasonable soft label is always better, always!!
loss = tf.losses.sparse_softmax_cross_entropy(labels=tf_y, logits=output)
accuracy = tf.metrics.accuracy(          # return (acc, update_op), and create 2 local variables
labels=tf.squeeze(tf_y), predictions=tf.argmax(output, axis=1),)[1]
optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.05)
train_op = optimizer.minimize(loss)

sess = tf.Session()
init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
sess.run(init_op)     # initialize var in graph

plt.ion()   # something about plotting
for step in range(100):
    # train and net output
    _, acc, pred = sess.run([train_op, accuracy, output], {tf_x: x, tf_y: y})
    if step % 20 == 0:
        # plot and show learning process
        plt.cla()
        plt.scatter(x[:, 0], x[:, 1], c=pred.argmax(1), s=100, lw=0, cmap='RdYlGn')
        plt.text(1.5, -4, 'Accuracy=%.2f' % acc, fontdict={'size': 20, 'color': 'red'})
        plt.pause(0.1)

plt.ioff()
plt.show()

mass2.gif

Common sense

  • Soft label provides more information image.png
  • Small learning rate is good for accuracy, and if you have no idea about this problem, smaller lr and adamoptimizer is your best choice

optimizer = tf.train.AdamOptimizer(learning_rate=0.0002)

  • All the neural networks are over-parameter, prune some parameters will help it trains faster and get better test performance
  • The gradient descent in every iterations or epochs is very importatnt, it can provide extra information.
  • Suitable loss function can imporve its performance at last, but it's not very important at the very begining.